package com.gome.han.bigdata.spark.core.rdd.dependency

import org.apache.spark.rdd.RDD
import org.apache.spark.{SparkConf, SparkContext}

/**
 * @author Hanpeng
 * @date 2021/1/15 21:03
 * @description:
 */
object DependencyDemo2 {
  def main(args: Array[String]): Unit = {
    val sparConf = new SparkConf().setMaster("local").setAppName("Dep")
    val sc = new SparkContext(sparConf)

    val lines: RDD[String] = sc.textFile("in/word.txt")
    //org.apache.spark.OneToOneDependency@48cd319d
    println(lines.dependencies)
    println("*************************")
    val words: RDD[String] = lines.flatMap(_.split(" "))
    println(words.dependencies)
    println("*************************")
    val wordToOne = words.map(word=>(word,1))
    //依赖关系
    println(wordToOne.dependencies)
    println("*************************")
    //val wordToSum: RDD[(String, Int)] = wordToOne.reduceByKey(_+_)
    //List(org.apache.spark.ShuffleDependency@2bfc2f8b)
    val wordToSum: RDD[(String, Int)] = wordToOne.reduceByKey((x,y)=>x+y)
    println(wordToSum.dependencies)
    println("*************************")
    val array: Array[(String, Int)] = wordToSum.collect()
    array.foreach(println)

    sc.stop()
  }
}
